{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "21f9146a-c6f3-4a78-b3fb-0d262492e87c",
   "metadata": {},
   "source": [
    "# Lesson 3: Sentence Window Retrieval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "541cadae-916c-42da-93ff-75d7f788ee8d",
   "metadata": {
    "height": 46
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "78f48098-16d8-4209-b722-1ec6a0220c96",
   "metadata": {
    "height": 97
   },
   "outputs": [],
   "source": [
    "import utils\n",
    "\n",
    "import os\n",
    "import openai\n",
    "openai.api_key = utils.get_openai_api_key()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9ee55360-1fc4-41cc-bd49-82027797ea40",
   "metadata": {
    "height": 97,
    "tags": []
   },
   "outputs": [],
   "source": [
    "from llama_index import SimpleDirectoryReader\n",
    "\n",
    "documents = SimpleDirectoryReader(\n",
    "    input_files=[\"./eBook-How-to-Build-a-Career-in-AI.pdf\"]\n",
    ").load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a12ada81-5c1c-47c9-b7b4-ba621a80bbcd",
   "metadata": {
    "height": 80,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'> \n",
      "\n",
      "41 \n",
      "\n",
      "<class 'llama_index.schema.Document'>\n",
      "Doc ID: 93473ac6-4e07-4e6d-a134-186916cd3440\n",
      "Text: PAGE 1Founder, DeepLearning.AICollected Insights from Andrew Ng\n",
      "How to  Build Your Career in AIA Simple Guide\n"
     ]
    }
   ],
   "source": [
    "print(type(documents), \"\\n\")\n",
    "print(len(documents), \"\\n\")\n",
    "print(type(documents[0]))\n",
    "print(documents[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f2f662e4-3fb8-40c6-acc5-e8510348d113",
   "metadata": {
    "height": 63,
    "tags": []
   },
   "outputs": [],
   "source": [
    "from llama_index import Document\n",
    "\n",
    "document = Document(text=\"\\n\\n\".join([doc.text for doc in documents]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "46ff01ea-b5b0-4e65-8565-b2444812bd84",
   "metadata": {},
   "source": [
    "## Window-sentence retrieval setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6a194b93-f975-4d38-babe-218d3aae6117",
   "metadata": {
    "height": 148,
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /tmp/llama_index...\n",
      "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
     ]
    }
   ],
   "source": [
    "from llama_index.node_parser import SentenceWindowNodeParser\n",
    "\n",
    "# create the sentence window node parser w/ default settings\n",
    "node_parser = SentenceWindowNodeParser.from_defaults(\n",
    "    window_size=3,\n",
    "    window_metadata_key=\"window\",\n",
    "    original_text_metadata_key=\"original_text\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "88973de6-be8f-4653-aca3-8d0e884d9470",
   "metadata": {
    "height": 63,
    "tags": []
   },
   "outputs": [],
   "source": [
    "text = \"hello. how are you? I am fine!  \"\n",
    "\n",
    "nodes = node_parser.get_nodes_from_documents([Document(text=text)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "860e40a8-6f50-4345-b314-c4d9349681c6",
   "metadata": {
    "height": 29,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['hello. ', 'how are you? ', 'I am fine!  ']\n"
     ]
    }
   ],
   "source": [
    "print([x.text for x in nodes])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b4334e02-c423-4555-8446-4794eadccd0a",
   "metadata": {
    "height": 29,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hello.  how are you?  I am fine!  \n"
     ]
    }
   ],
   "source": [
    "print(nodes[1].metadata[\"window\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "03fea59c-ed0f-4cc8-87b4-871798edb094",
   "metadata": {
    "height": 63,
    "tags": []
   },
   "outputs": [],
   "source": [
    "text = \"hello. foo bar. cat dog. mouse\"\n",
    "\n",
    "nodes = node_parser.get_nodes_from_documents([Document(text=text)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c329808d-248f-422e-bce0-1ac1ecba79a5",
   "metadata": {
    "height": 29,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['hello. ', 'foo bar. ', 'cat dog. ', 'mouse']\n"
     ]
    }
   ],
   "source": [
    "print([x.text for x in nodes])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "cfd1eccb-823d-4f6f-8d6c-a9064dc76922",
   "metadata": {
    "height": 29,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hello.  foo bar.  cat dog. \n"
     ]
    }
   ],
   "source": [
    "print(nodes[0].metadata[\"window\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fea97dda-8457-4f32-a2c7-26ae92eaf0b4",
   "metadata": {},
   "source": [
    "### Building the index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "bb2387dd-6bdb-4d0d-98ea-2b468ddfe160",
   "metadata": {
    "height": 63,
    "tags": []
   },
   "outputs": [],
   "source": [
    "from llama_index.llms import OpenAI\n",
    "\n",
    "llm = OpenAI(model=\"gpt-3.5-turbo\", temperature=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "5102d65b-3584-4a25-88be-7d5dbc70c678",
   "metadata": {
    "height": 148,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "881293e1722a40b6aa783826eb59da27",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/743 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cd25fbc6a2fe410193a841b1b3c81393",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/133M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f855fcbc8dd64c72901d69fad861c1b7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/366 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "42955117788c48c9bb5352ed4d4dff03",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "143b4216c2e44db9aadeb982571d40d0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/711k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2973975b84d148f8b8c3aa6f29ba2219",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "special_tokens_map.json:   0%|          | 0.00/125 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from llama_index import ServiceContext\n",
    "\n",
    "sentence_context = ServiceContext.from_defaults(\n",
    "    llm=llm,\n",
    "    embed_model=\"local:BAAI/bge-small-en-v1.5\",\n",
    "    # embed_model=\"local:BAAI/bge-large-en-v1.5\"\n",
    "    node_parser=node_parser,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "fbf15398-25e9-47bd-b840-e57272d38683",
   "metadata": {
    "height": 97,
    "tags": []
   },
   "outputs": [],
   "source": [
    "from llama_index import VectorStoreIndex\n",
    "\n",
    "sentence_index = VectorStoreIndex.from_documents(\n",
    "    [document], service_context=sentence_context\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "55497194-35c9-4f91-85c4-ef6adb1aff78",
   "metadata": {
    "height": 29,
    "tags": []
   },
   "outputs": [],
   "source": [
    "sentence_index.storage_context.persist(persist_dir=\"./sentence_index\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ba4e7371-7a8e-451c-a5e7-e9e3d7371614",
   "metadata": {
    "height": 335,
    "tags": []
   },
   "outputs": [],
   "source": [
    "# This block of code is optional to check\n",
    "# if an index file exist, then it will load it\n",
    "# if not, it will rebuild it\n",
    "\n",
    "import os\n",
    "from llama_index import VectorStoreIndex, StorageContext, load_index_from_storage\n",
    "from llama_index import load_index_from_storage\n",
    "\n",
    "if not os.path.exists(\"./sentence_index\"):\n",
    "    sentence_index = VectorStoreIndex.from_documents(\n",
    "        [document], service_context=sentence_context\n",
    "    )\n",
    "\n",
    "    sentence_index.storage_context.persist(persist_dir=\"./sentence_index\")\n",
    "else:\n",
    "    sentence_index = load_index_from_storage(\n",
    "        StorageContext.from_defaults(persist_dir=\"./sentence_index\"),\n",
    "        service_context=sentence_context\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b934879-e2c3-44f8-b98c-0300dc6389a9",
   "metadata": {},
   "source": [
    "### Building the postprocessor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "9f22f435-a90a-447c-9e63-8aebec1e968d",
   "metadata": {
    "height": 97,
    "tags": []
   },
   "outputs": [],
   "source": [
    "from llama_index.indices.postprocessor import MetadataReplacementPostProcessor\n",
    "\n",
    "postproc = MetadataReplacementPostProcessor(\n",
    "    target_metadata_key=\"window\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "77fad243-7ea8-4a81-b85e-91c3e638d932",
   "metadata": {
    "height": 97,
    "tags": []
   },
   "outputs": [],
   "source": [
    "from llama_index.schema import NodeWithScore\n",
    "from copy import deepcopy\n",
    "\n",
    "scored_nodes = [NodeWithScore(node=x, score=1.0) for x in nodes]\n",
    "nodes_old = [deepcopy(n) for n in nodes]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "946c4219-1d6e-4717-8c8f-40db659ea517",
   "metadata": {
    "height": 29,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'foo bar. '"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nodes_old[1].text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "89900cb4-5d03-43f6-9d1d-de1350bfd299",
   "metadata": {
    "height": 29,
    "tags": []
   },
   "outputs": [],
   "source": [
    "replaced_nodes = postproc.postprocess_nodes(scored_nodes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "72f98170-e368-461b-9939-5da80c4940e4",
   "metadata": {
    "height": 29,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hello.  foo bar.  cat dog.  mouse\n"
     ]
    }
   ],
   "source": [
    "print(replaced_nodes[1].text)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "72aa4a50-5c0b-4f23-9eae-ec67cb701e29",
   "metadata": {},
   "source": [
    "### Adding a reranker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "57608e01-ee96-4619-aab1-81d7fce1cbd1",
   "metadata": {
    "height": 131,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c82ede76519f463ba92aa141959b5609",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/799 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3db7c0601e3b48338e82b9a7348a30d4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/1.11G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02c03ff7a0d24a3c83a963838441b1e1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/443 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1acc24d76a1047de9fe10f7e8608522f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "sentencepiece.bpe.model:   0%|          | 0.00/5.07M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6576df3e89784cd5a3d6cb41b5e0d7e1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/17.1M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6274cd2216ee4f108e8ef6b81f3576d2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "special_tokens_map.json:   0%|          | 0.00/279 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from llama_index.indices.postprocessor import SentenceTransformerRerank\n",
    "\n",
    "# BAAI/bge-reranker-base\n",
    "# link: https://huggingface.co/BAAI/bge-reranker-base\n",
    "rerank = SentenceTransformerRerank(\n",
    "    top_n=2, model=\"BAAI/bge-reranker-base\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "4d89da07-6b0c-477f-bd0a-2f466c9b159b",
   "metadata": {
    "height": 165,
    "tags": []
   },
   "outputs": [],
   "source": [
    "from llama_index import QueryBundle\n",
    "from llama_index.schema import TextNode, NodeWithScore\n",
    "\n",
    "query = QueryBundle(\"I want a dog.\")\n",
    "\n",
    "scored_nodes = [\n",
    "    NodeWithScore(node=TextNode(text=\"This is a cat\"), score=0.6),\n",
    "    NodeWithScore(node=TextNode(text=\"This is a dog\"), score=0.4),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "1adc6cb4-e741-480a-ab13-06e9194b13b2",
   "metadata": {
    "height": 63,
    "tags": []
   },
   "outputs": [],
   "source": [
    "reranked_nodes = rerank.postprocess_nodes(\n",
    "    scored_nodes, query_bundle=query\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "3be841f6-0ab2-4c60-b52f-d27f81fcb1bb",
   "metadata": {
    "height": 29,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('This is a dog', 0.91827416), ('This is a cat', 0.0014040814)]\n"
     ]
    }
   ],
   "source": [
    "print([(x.text, x.score) for x in reranked_nodes])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74d51921-4b0b-4d89-963f-9a9e0ea439d9",
   "metadata": {},
   "source": [
    "### Runing the query engine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "47bfe5a5-5ce1-4baa-ba61-3b39d16a2337",
   "metadata": {
    "height": 63,
    "tags": []
   },
   "outputs": [],
   "source": [
    "sentence_window_engine = sentence_index.as_query_engine(\n",
    "    similarity_top_k=6, node_postprocessors=[postproc, rerank]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "c4f166b9-99f9-4507-af59-3347eaf596c6",
   "metadata": {
    "height": 63,
    "tags": []
   },
   "outputs": [],
   "source": [
    "window_response = sentence_window_engine.query(\n",
    "    \"What are the keys to building a career in AI?\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "535c536c",
   "metadata": {
    "height": 29
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Response(response='Learning foundational technical skills, working on projects, finding a job, and being part of a supportive community are the keys to building a career in AI.', source_nodes=[NodeWithScore(node=TextNode(id_='e2d22eca-ecbb-4f09-9f6a-6a68631b52b2', embedding=None, metadata={'window': 'Chapter 7: A Simple Framework for Starting Your AI \\nJob Search.\\n Chapter 8: Using Informational Interviews to Find \\nthe Right Job.\\n Chapter 9: Finding the Right AI Job for You.\\n Chapter 10: Keys to Building a Career in AI.\\n Chapter 11: Overcoming Imposter Syndrome.\\n Final Thoughts: Make Every Day Count.LEARNING\\nPROJECTS\\nJOB\\n\\nPAGE 4Coding AI Is the New Literacy\\nToday we take it for granted that many people know how to read and write. ', 'original_text': 'Chapter 10: Keys to Building a Career in AI.\\n'}, excluded_embed_metadata_keys=['window', 'original_text'], excluded_llm_metadata_keys=['window', 'original_text'], relationships={<NodeRelationship.SOURCE: '1'>: RelatedNodeInfo(node_id='882bb008-8b11-443b-8097-f5de9a46a9f1', node_type=<ObjectType.DOCUMENT: '4'>, metadata={}, hash='fdc7ce9a017ce62d4e0b30cc419051a9bb56618a8569f51a2b64f8535fe6244e'), <NodeRelationship.PREVIOUS: '2'>: RelatedNodeInfo(node_id='210af3a1-fc87-43d7-a4ec-88b6ded294f0', node_type=<ObjectType.TEXT: '1'>, metadata={'window': 'Chapter 6: Building a Portfolio of Projects that \\nShows Skill Progression.\\n Chapter 7: A Simple Framework for Starting Your AI \\nJob Search.\\n Chapter 8: Using Informational Interviews to Find \\nthe Right Job.\\n Chapter 9: Finding the Right AI Job for You.\\n Chapter 10: Keys to Building a Career in AI.\\n Chapter 11: Overcoming Imposter Syndrome.\\n', 'original_text': 'Chapter 9: Finding the Right AI Job for You.\\n'}, hash='85d37b2d35aa5f7eaaaedacdf54c1ed5dfae1a4d13de7dcae63a262bfea26848'), <NodeRelationship.NEXT: '3'>: RelatedNodeInfo(node_id='38618af5-136a-4e31-97b4-4ce9ed06063d', node_type=<ObjectType.TEXT: '1'>, metadata={'window': 'Chapter 8: Using Informational Interviews to Find \\nthe Right Job.\\n Chapter 9: Finding the Right AI Job for You.\\n Chapter 10: Keys to Building a Career in AI.\\n Chapter 11: Overcoming Imposter Syndrome.\\n Final Thoughts: Make Every Day Count.LEARNING\\nPROJECTS\\nJOB\\n\\nPAGE 4Coding AI Is the New Literacy\\nToday we take it for granted that many people know how to read and write.  Someday, I hope, \\nit will be just as common that people know how to write code, specifically for AI.\\n', 'original_text': 'Chapter 11: Overcoming Imposter Syndrome.\\n'}, hash='534e8c61067bcb57a0ba0541d926fc5487df566a558aba74825653713f8ad983')}, hash='dc02078636511b712569a3079c7b15c8d1406d2a99665c89d6b599f2785d44b2', text='Chapter 7: A Simple Framework for Starting Your AI \\nJob Search.\\n Chapter 8: Using Informational Interviews to Find \\nthe Right Job.\\n Chapter 9: Finding the Right AI Job for You.\\n Chapter 10: Keys to Building a Career in AI.\\n Chapter 11: Overcoming Imposter Syndrome.\\n Final Thoughts: Make Every Day Count.LEARNING\\nPROJECTS\\nJOB\\n\\nPAGE 4Coding AI Is the New Literacy\\nToday we take it for granted that many people know how to read and write. ', start_char_idx=802, end_char_idx=847, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'), score=0.9518099), NodeWithScore(node=TextNode(id_='87bd8ceb-18f8-40d1-b36c-211e81546386', embedding=None, metadata={'window': 'Many companies are \\nstill trying to figure out which AI skills they need, and how to hire people \\nwho have them.  Things you’ve worked on may be significantly different \\nthan anything your interviewer has seen, and you’re more likely to have to \\neducate potential employers about some elements of your work.Inconsistent opinions on AI skills and jobs roles: CHAPTER 1\\nAs you go through each step, you should also build a supportive community.  Having friends and \\nallies who can help you — and who you strive to help — makes the path easier.  This is true whether \\nyou’re taking your first steps or you’ve been on the journey for years.LEARNING\\nPROJECTS\\nJOB\\n\\nPAGE 8Learning Technical \\nSkills for a Promising \\nAI CareerCHAPTER 2\\nLEARNING\\n\\nPAGE 9In the previous chapter, I introduced three key steps for building a career in AI: learning \\nfoundational technical skills, working on projects, and finding a job, all of which is supported \\nby being part of a community.  In this chapter, I’d like to dive more deeply into the first step: \\nlearning foundational skills.\\n More research papers have been published on AI than anyone can read in a lifetime. ', 'original_text': 'This is true whether \\nyou’re taking your first steps or you’ve been on the journey for years.LEARNING\\nPROJECTS\\nJOB\\n\\nPAGE 8Learning Technical \\nSkills for a Promising \\nAI CareerCHAPTER 2\\nLEARNING\\n\\nPAGE 9In the previous chapter, I introduced three key steps for building a career in AI: learning \\nfoundational technical skills, working on projects, and finding a job, all of which is supported \\nby being part of a community. '}, excluded_embed_metadata_keys=['window', 'original_text'], excluded_llm_metadata_keys=['window', 'original_text'], relationships={<NodeRelationship.SOURCE: '1'>: RelatedNodeInfo(node_id='882bb008-8b11-443b-8097-f5de9a46a9f1', node_type=<ObjectType.DOCUMENT: '4'>, metadata={}, hash='fdc7ce9a017ce62d4e0b30cc419051a9bb56618a8569f51a2b64f8535fe6244e'), <NodeRelationship.PREVIOUS: '2'>: RelatedNodeInfo(node_id='bf1dce99-66f9-436a-8c9c-07854ea1c0de', node_type=<ObjectType.TEXT: '1'>, metadata={'window': 'Even after the system has hit the target, further iteration may \\nbe necessary to address post-deployment drift.Working on projects often means collaborating with \\nstakeholders who lack expertise in AI:\\nWhile searching for a job in AI can be similar to searching for a job in \\nother sectors, there are also important differences.  Many companies are \\nstill trying to figure out which AI skills they need, and how to hire people \\nwho have them.  Things you’ve worked on may be significantly different \\nthan anything your interviewer has seen, and you’re more likely to have to \\neducate potential employers about some elements of your work.Inconsistent opinions on AI skills and jobs roles: CHAPTER 1\\nAs you go through each step, you should also build a supportive community.  Having friends and \\nallies who can help you — and who you strive to help — makes the path easier.  This is true whether \\nyou’re taking your first steps or you’ve been on the journey for years.LEARNING\\nPROJECTS\\nJOB\\n\\nPAGE 8Learning Technical \\nSkills for a Promising \\nAI CareerCHAPTER 2\\nLEARNING\\n\\nPAGE 9In the previous chapter, I introduced three key steps for building a career in AI: learning \\nfoundational technical skills, working on projects, and finding a job, all of which is supported \\nby being part of a community.  In this chapter, I’d like to dive more deeply into the first step: \\nlearning foundational skills.\\n', 'original_text': 'Having friends and \\nallies who can help you — and who you strive to help — makes the path easier. '}, hash='243ce5fbe8fcee975d0acf3bb7db39b55ea6d6c1b26eb094ceb77ab7d6903f52'), <NodeRelationship.NEXT: '3'>: RelatedNodeInfo(node_id='3f67208e-733a-47b0-9b17-e2a9a86a815c', node_type=<ObjectType.TEXT: '1'>, metadata={'window': 'Things you’ve worked on may be significantly different \\nthan anything your interviewer has seen, and you’re more likely to have to \\neducate potential employers about some elements of your work.Inconsistent opinions on AI skills and jobs roles: CHAPTER 1\\nAs you go through each step, you should also build a supportive community.  Having friends and \\nallies who can help you — and who you strive to help — makes the path easier.  This is true whether \\nyou’re taking your first steps or you’ve been on the journey for years.LEARNING\\nPROJECTS\\nJOB\\n\\nPAGE 8Learning Technical \\nSkills for a Promising \\nAI CareerCHAPTER 2\\nLEARNING\\n\\nPAGE 9In the previous chapter, I introduced three key steps for building a career in AI: learning \\nfoundational technical skills, working on projects, and finding a job, all of which is supported \\nby being part of a community.  In this chapter, I’d like to dive more deeply into the first step: \\nlearning foundational skills.\\n More research papers have been published on AI than anyone can read in a lifetime.  So, when \\nlearning, it’s critical to prioritize topic selection. ', 'original_text': 'In this chapter, I’d like to dive more deeply into the first step: \\nlearning foundational skills.\\n'}, hash='bfc10ad686900db0eebbe251bd26eb1489bfee49558e4b9862e3b534a6adf934')}, hash='99f7fd2d5d7f8547d53460f606780bc545e6aa7fe8541c4d2a68c21fb8c1360b', text='Many companies are \\nstill trying to figure out which AI skills they need, and how to hire people \\nwho have them.  Things you’ve worked on may be significantly different \\nthan anything your interviewer has seen, and you’re more likely to have to \\neducate potential employers about some elements of your work.Inconsistent opinions on AI skills and jobs roles: CHAPTER 1\\nAs you go through each step, you should also build a supportive community.  Having friends and \\nallies who can help you — and who you strive to help — makes the path easier.  This is true whether \\nyou’re taking your first steps or you’ve been on the journey for years.LEARNING\\nPROJECTS\\nJOB\\n\\nPAGE 8Learning Technical \\nSkills for a Promising \\nAI CareerCHAPTER 2\\nLEARNING\\n\\nPAGE 9In the previous chapter, I introduced three key steps for building a career in AI: learning \\nfoundational technical skills, working on projects, and finding a job, all of which is supported \\nby being part of a community.  In this chapter, I’d like to dive more deeply into the first step: \\nlearning foundational skills.\\n More research papers have been published on AI than anyone can read in a lifetime. ', start_char_idx=6355, end_char_idx=6777, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'), score=0.9478973)], metadata={'e2d22eca-ecbb-4f09-9f6a-6a68631b52b2': {'window': 'Chapter 7: A Simple Framework for Starting Your AI \\nJob Search.\\n Chapter 8: Using Informational Interviews to Find \\nthe Right Job.\\n Chapter 9: Finding the Right AI Job for You.\\n Chapter 10: Keys to Building a Career in AI.\\n Chapter 11: Overcoming Imposter Syndrome.\\n Final Thoughts: Make Every Day Count.LEARNING\\nPROJECTS\\nJOB\\n\\nPAGE 4Coding AI Is the New Literacy\\nToday we take it for granted that many people know how to read and write. ', 'original_text': 'Chapter 10: Keys to Building a Career in AI.\\n'}, '87bd8ceb-18f8-40d1-b36c-211e81546386': {'window': 'Many companies are \\nstill trying to figure out which AI skills they need, and how to hire people \\nwho have them.  Things you’ve worked on may be significantly different \\nthan anything your interviewer has seen, and you’re more likely to have to \\neducate potential employers about some elements of your work.Inconsistent opinions on AI skills and jobs roles: CHAPTER 1\\nAs you go through each step, you should also build a supportive community.  Having friends and \\nallies who can help you — and who you strive to help — makes the path easier.  This is true whether \\nyou’re taking your first steps or you’ve been on the journey for years.LEARNING\\nPROJECTS\\nJOB\\n\\nPAGE 8Learning Technical \\nSkills for a Promising \\nAI CareerCHAPTER 2\\nLEARNING\\n\\nPAGE 9In the previous chapter, I introduced three key steps for building a career in AI: learning \\nfoundational technical skills, working on projects, and finding a job, all of which is supported \\nby being part of a community.  In this chapter, I’d like to dive more deeply into the first step: \\nlearning foundational skills.\\n More research papers have been published on AI than anyone can read in a lifetime. ', 'original_text': 'This is true whether \\nyou’re taking your first steps or you’ve been on the journey for years.LEARNING\\nPROJECTS\\nJOB\\n\\nPAGE 8Learning Technical \\nSkills for a Promising \\nAI CareerCHAPTER 2\\nLEARNING\\n\\nPAGE 9In the previous chapter, I introduced three key steps for building a career in AI: learning \\nfoundational technical skills, working on projects, and finding a job, all of which is supported \\nby being part of a community. '}})"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "window_response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "ada6304b-a86c-4f6c-a2e9-8ac5b1091a58",
   "metadata": {
    "height": 63
   },
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "**`Final Response:`** Learning foundational technical skills, working on projects, finding a job, and being part of a supportive community are the keys to building a career in AI."
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from llama_index.response.notebook_utils import display_response\n",
    "\n",
    "display_response(window_response)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cbcbe614-befe-4bf7-9813-2c91899650e9",
   "metadata": {},
   "source": [
    "## Putting it all Together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "313e8b1a-e9e7-4141-a4d2-72fb31a7e057",
   "metadata": {
    "height": 913
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from llama_index import ServiceContext, VectorStoreIndex, StorageContext\n",
    "from llama_index.node_parser import SentenceWindowNodeParser\n",
    "from llama_index.indices.postprocessor import MetadataReplacementPostProcessor\n",
    "from llama_index.indices.postprocessor import SentenceTransformerRerank\n",
    "from llama_index import load_index_from_storage\n",
    "\n",
    "\n",
    "def build_sentence_window_index(\n",
    "    documents,\n",
    "    llm,\n",
    "    embed_model=\"local:BAAI/bge-small-en-v1.5\",\n",
    "    sentence_window_size=3,\n",
    "    save_dir=\"sentence_index\",\n",
    "):\n",
    "    # create the sentence window node parser w/ default settings\n",
    "    node_parser = SentenceWindowNodeParser.from_defaults(\n",
    "        window_size=sentence_window_size,\n",
    "        window_metadata_key=\"window\",\n",
    "        original_text_metadata_key=\"original_text\",\n",
    "    )\n",
    "    sentence_context = ServiceContext.from_defaults(\n",
    "        llm=llm,\n",
    "        embed_model=embed_model,\n",
    "        node_parser=node_parser,\n",
    "    )\n",
    "    if not os.path.exists(save_dir):\n",
    "        sentence_index = VectorStoreIndex.from_documents(\n",
    "            documents, service_context=sentence_context\n",
    "        )\n",
    "        sentence_index.storage_context.persist(persist_dir=save_dir)\n",
    "    else:\n",
    "        sentence_index = load_index_from_storage(\n",
    "            StorageContext.from_defaults(persist_dir=save_dir),\n",
    "            service_context=sentence_context,\n",
    "        )\n",
    "\n",
    "    return sentence_index\n",
    "\n",
    "\n",
    "def get_sentence_window_query_engine(\n",
    "    sentence_index, similarity_top_k=6, rerank_top_n=2\n",
    "):\n",
    "    # define postprocessors\n",
    "    postproc = MetadataReplacementPostProcessor(target_metadata_key=\"window\")\n",
    "    rerank = SentenceTransformerRerank(\n",
    "        top_n=rerank_top_n, model=\"BAAI/bge-reranker-base\"\n",
    "    )\n",
    "\n",
    "    sentence_window_engine = sentence_index.as_query_engine(\n",
    "        similarity_top_k=similarity_top_k, node_postprocessors=[postproc, rerank]\n",
    "    )\n",
    "    return sentence_window_engine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3b5bb36f-97f9-4e03-bd4e-3ceb635a868b",
   "metadata": {
    "height": 148
   },
   "outputs": [],
   "source": [
    "from llama_index.llms import OpenAI\n",
    "\n",
    "index = build_sentence_window_index(\n",
    "    [document],\n",
    "    llm=OpenAI(model=\"gpt-3.5-turbo\", temperature=0.1),\n",
    "    save_dir=\"./sentence_index\",\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d0e78e0-7765-4037-a5fa-63b711dab5aa",
   "metadata": {
    "height": 46
   },
   "outputs": [],
   "source": [
    "query_engine = get_sentence_window_query_engine(index, similarity_top_k=6)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eb855430-8fcb-4c25-8d83-a30160449acf",
   "metadata": {},
   "source": [
    "## TruLens Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "07aa5320-5c1c-4636-aa72-e6a786a58c8a",
   "metadata": {
    "height": 114
   },
   "outputs": [],
   "source": [
    "eval_questions = []\n",
    "with open('generated_questions.text', 'r') as file:\n",
    "    for line in file:\n",
    "        # Remove newline character and convert to integer\n",
    "        item = line.strip()\n",
    "        eval_questions.append(item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f9bdeb7-8c37-46b0-80f2-eb8ef1cc7b59",
   "metadata": {
    "height": 114
   },
   "outputs": [],
   "source": [
    "from trulens_eval import Tru\n",
    "\n",
    "def run_evals(eval_questions, tru_recorder, query_engine):\n",
    "    for question in eval_questions:\n",
    "        with tru_recorder as recording:\n",
    "            response = query_engine.query(question)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7818a9b-04cc-4a6e-98ae-d3d0ab309998",
   "metadata": {
    "height": 97
   },
   "outputs": [],
   "source": [
    "from utils import get_prebuilt_trulens_recorder\n",
    "\n",
    "from trulens_eval import Tru\n",
    "\n",
    "Tru().reset_database()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "59ef4d27-2c4e-4dd0-85ed-a69a0f8eea00",
   "metadata": {},
   "source": [
    "### Sentence window size = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93e6c651-838d-4c36-b806-efd0d8c2d5f0",
   "metadata": {
    "height": 131
   },
   "outputs": [],
   "source": [
    "sentence_index_1 = build_sentence_window_index(\n",
    "    documents,\n",
    "    llm=OpenAI(model=\"gpt-3.5-turbo\", temperature=0.1),\n",
    "    embed_model=\"local:BAAI/bge-small-en-v1.5\",\n",
    "    sentence_window_size=1,\n",
    "    save_dir=\"sentence_index_1\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8dc445f4-0311-4192-ac2e-4ea94b2653e5",
   "metadata": {
    "height": 63
   },
   "outputs": [],
   "source": [
    "sentence_window_engine_1 = get_sentence_window_query_engine(\n",
    "    sentence_index_1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2750e2b1-ca60-495f-8168-33520740916c",
   "metadata": {
    "height": 80
   },
   "outputs": [],
   "source": [
    "tru_recorder_1 = get_prebuilt_trulens_recorder(\n",
    "    sentence_window_engine_1,\n",
    "    app_id='sentence window engine 1'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca942b16-52be-4107-861c-33daeca1f619",
   "metadata": {
    "height": 29
   },
   "outputs": [],
   "source": [
    "run_evals(eval_questions, tru_recorder_1, sentence_window_engine_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "55dc7629-7fb7-4801-bd52-c9ab0da50267",
   "metadata": {
    "height": 29
   },
   "outputs": [],
   "source": [
    "Tru().run_dashboard()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40d36ca4-8f52-4510-bfab-aa00b704d179",
   "metadata": {},
   "source": [
    "### Note about the dataset of questions\n",
    "- Since this evaluation process takes a long time to run, the following file `generated_questions.text` contains one question (the one mentioned in the lecture video).\n",
    "- If you would like to explore other possible questions, feel free to explore the file directory by clicking on the \"Jupyter\" logo at the top right of this notebook. You'll see the following `.text` files:\n",
    "\n",
    "> - `generated_questions_01_05.text`\n",
    "> - `generated_questions_06_10.text`\n",
    "> - `generated_questions_11_15.text`\n",
    "> - `generated_questions_16_20.text`\n",
    "> - `generated_questions_21_24.text`\n",
    "\n",
    "Note that running an evaluation on more than one question can take some time, so we recommend choosing one of these files (with 5 questions each) to run and explore the results.\n",
    "\n",
    "- For evaluating a personal project, an eval set of 20 is reasonable.\n",
    "- For evaluating business applications, you may need a set of 100+ in order to cover all the use cases thoroughly.\n",
    "- Note that since API calls can sometimes fail, you may occasionally see null responses, and would want to re-run your evaluations.  So running your evaluations in smaller batches can also help you save time and cost by only re-running the evaluation on the batches with issues."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c55e4e1-5f1f-4c50-bd8e-4b54299377da",
   "metadata": {
    "height": 114
   },
   "outputs": [],
   "source": [
    "eval_questions = []\n",
    "with open('generated_questions.text', 'r') as file:\n",
    "    for line in file:\n",
    "        # Remove newline character and convert to integer\n",
    "        item = line.strip()\n",
    "        eval_questions.append(item)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e54c9977-c606-41bb-b87a-ec3cc17eabaf",
   "metadata": {},
   "source": [
    "### Sentence window size = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c8bea9d-9805-4302-b8c0-6e0f3eeec030",
   "metadata": {
    "height": 267
   },
   "outputs": [],
   "source": [
    "sentence_index_3 = build_sentence_window_index(\n",
    "    documents,\n",
    "    llm=OpenAI(model=\"gpt-3.5-turbo\", temperature=0.1),\n",
    "    embed_model=\"local:BAAI/bge-small-en-v1.5\",\n",
    "    sentence_window_size=3,\n",
    "    save_dir=\"sentence_index_3\",\n",
    ")\n",
    "sentence_window_engine_3 = get_sentence_window_query_engine(\n",
    "    sentence_index_3\n",
    ")\n",
    "\n",
    "tru_recorder_3 = get_prebuilt_trulens_recorder(\n",
    "    sentence_window_engine_3,\n",
    "    app_id='sentence window engine 3'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e69d459e-e984-46b4-8fd9-ca4f77150647",
   "metadata": {
    "height": 29
   },
   "outputs": [],
   "source": [
    "run_evals(eval_questions, tru_recorder_3, sentence_window_engine_3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dfdfb770",
   "metadata": {
    "height": 29
   },
   "outputs": [],
   "source": [
    "Tru().run_dashboard()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
